import scrapy
import re
import json,jsonpath
from scrapy import Selector
from ..items import XiechengItem

class Tourism_Info(scrapy.Spider):
    name='xiecheng'
    def __init__(self,keyword):
        '''
        param keyword : 需要搜索的关键词
        '''
        self.keyword = keyword
        self.start_urls = ['http://vacations.ctrip.com/whole-17B126/?searchValue=%s&searchText=%s&from=do'%(self.keyword,self.keyword)]
        self.product = 'http://vacations.ctrip.com/tour-mainsite-vacations/api/product'
    def parse(self,response):
        '''
        得到 id list
        return list
        '''
        html = Selector(text=response.body)
        #ids = html.re('data-params=\'{"Id": (.*?),')
        #tpls =  html.re('"tpl":"(.*?)",')
        data_params = html.re('data-params=\'(.*?)\'')
        #dis = html.re('"Did":(.*?),')
        #urls = html.re('"Url":"(.*?)"}\'')
        list=[]
        dis = {}
        for i in data_params:
            j = json.loads(i)
            item={
                'Id': j['Id'],
                'Bu': j['Bu'],
                'Did':j['Did']
            }
            list.append(item)
            dis[j['Id']] = j['Url']
        unicornHeader = {
            'Referer': 'http://vacations.ctrip.com/tours/d-beijing-1?from=do',
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36',
            'X-Requested-With': 'XMLHttpRequest'
        }
        yield scrapy.FormRequest(
            self.product,
            headers=unicornHeader,
            method = 'POST',
            meta = dis,
            formdata={
                 "params":str(list),
                 "keyword":self.keyword,
             },
            callback=self.parse_info
         )
        next_page = response.xpath("//div[@id='_pg']/a[@class='down ']/@href").extract_first()
        if next_page is not None:
            next_page = response.urljoin(next_page)
            yield scrapy.Request(next_page, callback=self.parse)
    def parse_info(self,response):
        j = json.loads(response.body)
        for items in j :
            item = XiechengItem()
            item['score'] = items['Score']
            item['travelCount'] = items['TravelCount']
            item['proName'] = items['ProName'].strip()
            item['price'] = re.sub(r'\D','',str(items['Price'])).strip()
            item['vendorName'] = items['VendorName'].strip()
            item['departureCityName'] = items['DepartureCityName'].strip()
            item['dsestinationName'] = items['DestinationName'].strip()
            item['url'] = response.meta[items['ProductId']]
            yield item
        